{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concatenate all files\n",
    "\n",
    "```bash\n",
    "$ cd path/to/train-easy/\n",
    "$ find -name '*.txt' -exec cat {} \\; > ../../../interim/train-easy_all.txt\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0802 13:37:37.702202 4617815488 deprecation_wrapper.py:118] From /Users/lewtun/git/deep-math/notebooks/baselines/lstm.py:7: The name tf.keras.layers.CuDNNLSTM is deprecated. Please use tf.compat.v1.keras.layers.CuDNNLSTM instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0-beta1\n",
      "GPU Available:  False\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import glob\n",
    "import pickle\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "from lstm import LSTM_S2S\n",
    "from metrics import exact_match_metric_index\n",
    "from callbacks import NValidationSetsCallback, GradientLogger\n",
    "import sys\n",
    "sys.path.append('../../')\n",
    "from src.models.generator import DataGeneratorAttention\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.layers import Bidirectional, Layer, Input, Dense, LSTM, Embedding, Activation, dot, concatenate, TimeDistributed\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.compat.v1.keras.layers import CuDNNLSTM\n",
    "print(tf.__version__)\n",
    "print(\"GPU Available: \", tf.test.is_gpu_available())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "settings_path = Path('../../settings/settings_local.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(str(settings_path), 'r') as file:\n",
    "    settings_dict = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'math_module': 'numbers__round_number',\n",
       " 'train_level': 'easy',\n",
       " 'batch_size': 1024,\n",
       " 'thinking_steps': 0,\n",
       " 'epochs': 1,\n",
       " 'latent_dim': 256,\n",
       " 'save_path': '/artifacts/',\n",
       " 'data_path': '../../data/raw/v1.0/'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "settings_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data\n",
    "\n",
    "Start with batching a single file before tackling the whole dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36mextrapolate\u001b[m\u001b[m  \u001b[1m\u001b[36minterpolate\u001b[m\u001b[m  \u001b[1m\u001b[36mtrain-easy\u001b[m\u001b[m   \u001b[1m\u001b[36mtrain-hard\u001b[m\u001b[m   \u001b[1m\u001b[36mtrain-medium\u001b[m\u001b[m\n"
     ]
    }
   ],
   "source": [
    "raw_path = Path(settings_dict['data_path'])\n",
    "!ls {raw_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "algebra__linear_1d.txt\n",
      "algebra__linear_1d_composed.txt\n",
      "algebra__linear_2d.txt\n",
      "algebra__linear_2d_composed.txt\n",
      "algebra__polynomial_roots.txt\n"
     ]
    }
   ],
   "source": [
    "interpolate_path = raw_path/'interpolate'\n",
    "!ls {interpolate_path} | head -5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "algebra__polynomial_roots_big.txt\n",
      "arithmetic__add_or_sub_big.txt\n",
      "arithmetic__add_sub_multiple_longer.txt\n",
      "arithmetic__div_big.txt\n",
      "arithmetic__mixed_longer.txt\n"
     ]
    }
   ],
   "source": [
    "extrapolate_path = raw_path/'extrapolate'\n",
    "!ls {extrapolate_path} | head -5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "algebra__linear_1d.txt\n",
      "algebra__linear_1d_composed.txt\n",
      "algebra__linear_2d.txt\n",
      "algebra__linear_2d_composed.txt\n",
      "algebra__polynomial_roots.txt\n"
     ]
    }
   ],
   "source": [
    "train_easy_path = raw_path/'train-easy/'\n",
    "!ls {train_easy_path} | head -5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concatenate_texts(path, pattern):\n",
    "    file_paths = list(path.glob('{}*.txt'.format(pattern)))\n",
    "    \n",
    "    input_texts = []\n",
    "    target_texts = []\n",
    "\n",
    "    for file_path in file_paths:\n",
    "        with open(str(file_path), 'r', encoding='utf-8') as f:\n",
    "            lines = f.read().split('\\n')[:-1]\n",
    "\n",
    "        input_texts.extend(lines[0::2])\n",
    "        target_texts.extend(['\\t' + target_text + '\\n' for target_text in lines[1::2]])\n",
    "        \n",
    "    return input_texts, target_texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "math_module = settings_dict[\"math_module\"]\n",
    "train_level = settings_dict[\"train_level\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = {\n",
    "    'train':(raw_path, 'train-' + train_level + '/' + math_module),\n",
    "    'interpolate':(interpolate_path, math_module),\n",
    "    'extrapolate':(extrapolate_path, math_module)\n",
    "           }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of set train is 1333332\n",
      "Length of set interpolate is 20000\n",
      "Length of set extrapolate is 10000\n",
      "CPU times: user 617 ms, sys: 240 ms, total: 857 ms\n",
      "Wall time: 864 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "input_texts = {}\n",
    "target_texts = {}\n",
    "\n",
    "for k, v in datasets.items():\n",
    "    input_texts[k], target_texts[k] = concatenate_texts(v[0], v[1])\n",
    "    print('Length of set {} is {}'.format(k, len(input_texts[k])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Sample:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INPUT: Round 0.000001671 to 7 dps.\n",
      "OUTPUT: 0.0000017\n"
     ]
    }
   ],
   "source": [
    "random_idx = np.random.randint(1, len(input_texts['train']))\n",
    "print('INPUT:', input_texts['train'][random_idx])\n",
    "print('OUTPUT:', target_texts['train'][random_idx].strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Concatenate texts to get text metrics (max length, number of unique tokens, etc.):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_input_texts = sum(input_texts.values(), [])\n",
    "all_target_texts = sum(target_texts.values(), [])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_characters = set(''.join(all_input_texts))\n",
    "target_characters = set(''.join(all_target_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples: 1363332\n",
      "Number of unique input tokens: 51\n",
      "Number of unique output tokens: 14\n",
      "Max sequence length for inputs: 160\n",
      "Max sequence length for outputs: 16\n"
     ]
    }
   ],
   "source": [
    "input_characters = sorted(list(input_characters))\n",
    "target_characters = sorted(list(target_characters))\n",
    "num_encoder_tokens = len(input_characters)\n",
    "num_decoder_tokens = len(target_characters)\n",
    "max_encoder_seq_length = max([len(txt) for txt in all_input_texts])\n",
    "max_decoder_seq_length = max([len(txt) for txt in all_target_texts])\n",
    "\n",
    "print('Number of samples:', len(all_input_texts))\n",
    "print('Number of unique input tokens:', num_encoder_tokens)\n",
    "print('Number of unique output tokens:', num_decoder_tokens)\n",
    "print('Max sequence length for inputs:', max_encoder_seq_length)\n",
    "print('Max sequence length for outputs:', max_decoder_seq_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Delete all texts to realease memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "del all_input_texts\n",
    "del all_target_texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create train test splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_texts_train, input_texts_valid, target_texts_train, target_texts_valid = train_test_split(input_texts['train'], target_texts['train'], test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training samples: 1066665\n"
     ]
    }
   ],
   "source": [
    "print('Number of training samples:', len(input_texts_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of validation samples: 266667\n"
     ]
    }
   ],
   "source": [
    "print('Number of validation samples:', len(input_texts_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorise the text\n",
    "Before training, we need to map strings to a numerical representation. Create two lookup tables: one mapping question characters to numbers, and another for answer characters to number."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a mapping from unique characters to indices\n",
    "input_token_index = dict([(char, i) for i, char in enumerate(input_characters)])\n",
    "target_token_index = dict([(char, i) for i, char in enumerate(target_characters)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create keras data generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters\n",
    "params = {'batch_size': settings_dict[\"batch_size\"],\n",
    "          'max_encoder_seq_length': max_encoder_seq_length,\n",
    "          'max_decoder_seq_length': max_decoder_seq_length,\n",
    "          'num_encoder_tokens': num_encoder_tokens,\n",
    "          'num_decoder_tokens': num_decoder_tokens,\n",
    "          'input_token_index': input_token_index,\n",
    "          'target_token_index': target_token_index,\n",
    "          'num_thinking_steps': settings_dict[\"thinking_steps\"]\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_generator = DataGeneratorAttention(input_texts=input_texts_train, target_texts=target_texts_train, **params)\n",
    "validation_generator = DataGeneratorAttention(input_texts=input_texts_valid, target_texts=target_texts_valid, **params)\n",
    "interpolate_generator = DataGeneratorAttention(input_texts=input_texts['interpolate'], target_texts=target_texts['interpolate'], **params)\n",
    "extrapolate_generator = DataGeneratorAttention(input_texts=input_texts['extrapolate'], target_texts=target_texts['extrapolate'], **params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_idx = 0\n",
    "# example_input_batch, example_target_batch = training_generator[example_idx][0][0][:,:,0], training_generator[example_idx][0][1][:,:,0]\n",
    "# example_input_batch.shape, example_target_batch.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[22., 30., 45., ...,  0.,  0.,  0.],\n",
       "       [25., 33., 26., ...,  0.,  0.,  0.],\n",
       "       [23., 40., 46., ...,  0.,  0.,  0.],\n",
       "       ...,\n",
       "       [25., 33., 26., ...,  0.,  0.,  0.],\n",
       "       [25., 33., 26., ...,  0.,  0.,  0.],\n",
       "       [23., 40., 46., ...,  0.,  0.,  0.]], dtype=float32)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_generator[example_idx][0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1024, 160)\n",
      "(1024, 16)\n",
      "(1024, 16, 14)\n"
     ]
    }
   ],
   "source": [
    "print(training_generator[example_idx][0][0].shape)\n",
    "print(training_generator[example_idx][0][1].shape)\n",
    "print(training_generator[example_idx][1].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_inputs = Input(shape=(max_encoder_seq_length,))\n",
    "decoder_inputs = Input(shape=(max_decoder_seq_length,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([None, 16])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder_inputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"embedding_4/Identity:0\", shape=(None, 160, 64), dtype=float32)\n",
      "Tensor(\"lstm_4/Identity:0\", shape=(None, 160, 256), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "encoder = Embedding(num_encoder_tokens + 1, 64, input_length=max_encoder_seq_length, mask_zero=True)(encoder_inputs)\n",
    "print(encoder)\n",
    "encoder_outputs = LSTM(256, return_sequences=True, unroll=True)(encoder)\n",
    "print(encoder_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoder Tensor(\"strided_slice_3:0\", shape=(None, 256), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "encoder_last = encoder_outputs[:,-1,:]\n",
    "encoder_last.set_shape([None, 256])\n",
    "print('encoder', encoder_last)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"embedding_11/Identity:0\", shape=(None, 16, 64), dtype=float32)\n",
      "decoder Tensor(\"lstm_11/Identity:0\", shape=(None, 16, 256), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "decoder = Embedding(num_decoder_tokens + 1, 64, input_length=max_decoder_seq_length, mask_zero=True)(decoder_inputs)\n",
    "print(decoder)\n",
    "decoder_outputs = LSTM(256, return_sequences=True, unroll=True)(decoder, initial_state=[encoder_last, encoder_last])\n",
    "\n",
    "print('decoder', decoder_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "attention Tensor(\"dot_4/Identity:0\", shape=(None, 16, 160), dtype=float32)\n",
      "attention Tensor(\"attention_2/Identity:0\", shape=(None, 16, 160), dtype=float32)\n",
      "context Tensor(\"dot_5/Identity:0\", shape=(None, 16, 256), dtype=float32)\n",
      "decoder_combined_context Tensor(\"concatenate_2/Identity:0\", shape=(None, 16, 512), dtype=float32)\n",
      "output 1 Tensor(\"time_distributed_4/Identity:0\", shape=(None, 16, 64), dtype=float32)\n",
      "output Tensor(\"time_distributed_5/Identity:0\", shape=(None, 16, 14), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Equation (7) with 'dot' score from Section 3.1 in the paper.\n",
    "# Note that we reuse Softmax-activation layer instead of writing tensor calculation\n",
    "attention = dot([decoder_outputs, encoder_outputs], axes=[2, 2])\n",
    "print('attention', attention)\n",
    "attention = Activation('softmax', name='attention')(attention)\n",
    "print('attention', attention)\n",
    "\n",
    "context = dot([attention, encoder_outputs], axes=[2,1])\n",
    "print('context', context)\n",
    "\n",
    "decoder_combined_context = concatenate([context, decoder_outputs])\n",
    "print('decoder_combined_context', decoder_combined_context)\n",
    "\n",
    "# # Has another weight + tanh layer as described in equation (5) of the paper\n",
    "output = TimeDistributed(Dense(64, activation=\"tanh\"))(decoder_combined_context)\n",
    "print('output 1',output)\n",
    "output = TimeDistributed(Dense(num_decoder_tokens, activation=\"softmax\"))(output)\n",
    "print('output', output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_dict = {\n",
    "    'validation':validation_generator,\n",
    "    'interpolation': interpolate_generator,\n",
    "    'extrapolation': extrapolate_generator\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = NValidationSetsCallback(valid_dict)\n",
    "gradient = GradientLogger(live_metrics=['loss', 'exact_match_metric_index'], live_gaps=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = settings_dict['epochs']  # Number of epochs to train for.\n",
    "latent_dim = settings_dict['latent_dim']  # Latent dimensionality of the encoding space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=[output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 16)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            [(None, 160)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 16, 64)       960         input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 160, 64)      3328        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 16, 64)       33024       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm (LSTM)                     (None, 160, 64)      33024       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dot (Dot)                       (None, 16, 160)      0           lstm_1[0][0]                     \n",
      "                                                                 lstm[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "attention (Activation)          (None, 16, 160)      0           dot[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 16, 64)       0           attention[0][0]                  \n",
      "                                                                 lstm[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 16, 128)      0           dot_1[0][0]                      \n",
      "                                                                 lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed (TimeDistribut (None, 16, 64)       8256        concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 16, 14)       910         time_distributed[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 79,502\n",
      "Trainable params: 79,502\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "adam = Adam(lr=6e-4, beta_1=0.9, beta_2=0.995, epsilon=1e-9, decay=0.0, amsgrad=False, clipnorm=0.1)\n",
    "\n",
    "model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=[exact_match_metric_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start training...\n",
      "{\"chart\": \"live_loss\", \"axis\": \"batch\"}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"axis\": \"batch\"}\n",
      "{\"chart\": \"loss\", \"axis\": \"epoch\"}\n",
      "{\"chart\": \"exact_match_metric_index\", \"axis\": \"epoch\"}\n",
      "{\"chart\": \"live_loss\", \"y\": 5.4358835, \"x\": 10}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6963021, \"x\": 10}\n",
      "{\"chart\": \"live_loss\", \"y\": 4.521158, \"x\": 20}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.69636446, \"x\": 20}\n",
      "{\"chart\": \"live_loss\", \"y\": 4.378515, \"x\": 30}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6968709, \"x\": 30}\n",
      "{\"chart\": \"live_loss\", \"y\": 4.3312016, \"x\": 40}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6967309, \"x\": 40}\n",
      "{\"chart\": \"live_loss\", \"y\": 4.27552, \"x\": 50}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6966175, \"x\": 50}\n",
      "{\"chart\": \"live_loss\", \"y\": 4.26984, \"x\": 60}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.69640714, \"x\": 60}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.9274971, \"x\": 70}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6964843, \"x\": 70}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.9456766, \"x\": 80}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6960763, \"x\": 80}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.8373485, \"x\": 90}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.69590926, \"x\": 90}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.771475, \"x\": 100}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6959569, \"x\": 100}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.6182945, \"x\": 110}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6960256, \"x\": 110}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.5625408, \"x\": 120}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6960066, \"x\": 120}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.4649978, \"x\": 130}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.69611055, \"x\": 130}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.4192574, \"x\": 140}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.69622695, \"x\": 140}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.3497348, \"x\": 150}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6962857, \"x\": 150}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.2365263, \"x\": 160}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.6962797, \"x\": 160}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.174389, \"x\": 170}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.69630915, \"x\": 170}\n",
      "{\"chart\": \"live_loss\", \"y\": 3.1387706, \"x\": 180}\n",
      "{\"chart\": \"live_exact_match_metric_index\", \"y\": 0.69626546, \"x\": 180}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-30731a88701a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m                                  \u001b[0;31m#use_multiprocessing=True, workers=8,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m                                  \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m                                  \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m                                 )\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1174\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1176\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m   1177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1178\u001b[0m   def evaluate_generator(self,\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m       \u001b[0mis_deferred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_compiled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 264\u001b[0;31m       \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    265\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[1;32m    916\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_sample_weight_modes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    917\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 918\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    919\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    920\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3508\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3509\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3510\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3511\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3512\u001b[0m     \u001b[0;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    570\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[1;32m    571\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[0;32m--> 572\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    574\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args)\u001b[0m\n\u001b[1;32m    669\u001b[0m     \u001b[0;31m# Only need to override the gradient in graph mode and when we have outputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    670\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 671\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_inference_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    672\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    673\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_register_gradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args)\u001b[0m\n\u001b[1;32m    443\u001b[0m             attrs=(\"executor_type\", executor_type,\n\u001b[1;32m    444\u001b[0m                    \"config_proto\", config),\n\u001b[0;32m--> 445\u001b[0;31m             ctx=ctx)\n\u001b[0m\u001b[1;32m    446\u001b[0m       \u001b[0;31m# Replace empty list with None\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/git/deep-math/env/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "print('start training...')\n",
    "train_hist = model.fit_generator(training_generator,\n",
    "                                 epochs=epochs,\n",
    "                                 #use_multiprocessing=True, workers=8,\n",
    "                                 callbacks=[history, gradient],\n",
    "                                 verbose=0,\n",
    "                                )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
